/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package regulatoryNetworks;

import functionalEnrichment.semanticSimilarity.semanticSimilarityCalculator.parsers.GoaParser;
import functionalEnrichment.semanticSimilarity.semanticSimilarityCalculator.parsers.OboParser;
import functionalEnrichment.semanticSimilarity.semanticSimilarityCalculator.units.AnnotatedGene;
import functionalEnrichment.semanticSimilarity.semanticSimilarityCalculator.units.GoTerm;
import functionalEnrichment.semanticSimilarity.semanticSimilarityCalculator.units.GoTermPair;
import functionalEnrichment.semanticSimilarity.semanticSimilarityCalculator.units.Ontology;
import functionalEnrichment.semanticSimilarity.semanticSimilarityCalculator.units.OntologyType;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.logging.Logger;
import jsc.independentsamples.MannWhitneyTest;
import jsc.independentsamples.TwoSampleTtest;
import jsc.onesample.Ttest;
import jsc.tests.H1;
import org.uncommons.maths.random.MersenneTwisterRNG;
import units.CellType;
import units.MonkeyType;
import utils.FileUtils;
import utils.Handle;

/**
 *
 * @author jamie
 */
public class GenePairTest {

    private File oboFile = new File("/Users/jamie/Dropbox/monkeys/GO/annotations/gene_ontology.obo");
    private File goaFile = new File("/Users/jamie/Dropbox/monkeys/GO/annotations/gene_association.goa_human");
    private File outputFile = new File("/Users/jamie/Dropbox/monkeys/GO/miTest/");
    private Map<String, Double> minetResults_AL;
    private Map<String, Double> minetResults_AP;
    private Map<String, Double> minetResults_ML;
    private Map<String, Double> minetResults_MP;
    private List<String> probeIds_AL;
    private List<String> probeIds_AP;
    private List<String> probeIds_ML;
    private List<String> probeIds_MP;
    private Handle handle = Handle.getConnectedHandle("monkeys");
    private Map<String, List<String>> swissToProbe;
    private Map<String, List<String>> probeToSwiss;
    private TreeMap<String, List<GoTerm>> goTermMap;
    private TreeSet<String> nonredundantSwissIds;
    private OntologyType ontologyType;
    private double difThreshold = 0.1;
    private File randFile = new File("/Users/jamie/Dropbox/monkeys/miTest/randSample.txt");
    private File realFile = new File("/Users/jamie/Dropbox/monkeys/miTest/results.txt");
    private File goResultsFile = new File("/Users/jamie/Documents/monkeys/miModel/goTermPairResults.txt");
    private File goResultsFile_model = new File("/Users/jamie/Documents/monkeys/miModel/model/");
    private File goResultsFile_pvalues = new File("/Users/jamie/Documents/monkeys/miModel/pValues.txt");
    double[] difs = null;
    private int minNoMiValuesPerTermPair = 3;
    private boolean limitToImmuneAndCellCycle = true;

    public void runTest(OntologyType ot) {

        this.ontologyType = ot;
        OboParser oboParser = new OboParser();
        Ontology ontology_mf = oboParser.getOntology(oboFile, ontologyType);
        GoaParser goaParser = new GoaParser(goaFile, ontology_mf);
        Map<String, AnnotatedGene> annotatedGenes = goaParser.getAnnotatedGenes();
        MinetResultsParser minetResultsParser;

        BufferedWriter writer = FileUtils.getBufferedWriter(realFile);
        try {
            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_AL = minetResultsParser.getResultsFromFile(MonkeyType.A, CellType.L);
            probeIds_AL = minetResultsParser.getProbeIds();

            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_AP = minetResultsParser.getResultsFromFile(MonkeyType.A, CellType.P);
            probeIds_AP = minetResultsParser.getProbeIds();

            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_ML = minetResultsParser.getResultsFromFile(MonkeyType.M, CellType.L);
            probeIds_ML = minetResultsParser.getProbeIds();

            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_MP = minetResultsParser.getResultsFromFile(MonkeyType.M, CellType.P);
            probeIds_MP = minetResultsParser.getProbeIds();
        } catch (IOException ex) {
            ex.printStackTrace();
        }

        nonredundantSwissIds = getNonRedundantSwissIds();
        swissToProbe = getSwissIdToProbeId();
        probeToSwiss = getProbeIdToSwiss();
        goTermMap = getGoTermMap(annotatedGenes);


        List<String> probeIds_intersect = new ArrayList<String>();
        probeIds_intersect.addAll(probeIds_AL);
        probeIds_intersect.retainAll(probeIds_AP);
        probeIds_intersect.retainAll(probeIds_ML);
        probeIds_intersect.retainAll(probeIds_MP);
//        System.out.println(probeIds_intersect.size());

        probeIds_intersect = filterList(probeIds_intersect);
        if (difs != null) {
            Arrays.sort(difs);
        }
        for (int i = 0; i < probeIds_intersect.size(); i++) {
            for (int j = i + 1; j < probeIds_intersect.size(); j++) {
                String[] ids = {probeIds_intersect.get(i), probeIds_intersect.get(j)};
                Arrays.sort(ids);
                String key = ids[0] + ":" + ids[1];
                double[] mi_A = {minetResults_AL.get(key), minetResults_AP.get(key)};
                double[] mi_M = {minetResults_ML.get(key), minetResults_MP.get(key)};
                double mean_A = (mi_A[0] + mi_A[1]) / 2;
                double mean_M = (mi_M[0] + mi_M[1]) / 2;
                double dif = 0;
                if (mean_A > mean_M) {
                    dif = mean_A - mean_M;
                } else {
                    dif = mean_M - mean_A;
                }

                double p = 1.0;
                if (difs != null) {
                    p = getP(difs, dif);
                }

                try {
                    writer.append(key + "\t" + mean_A + "\t" + mean_M + "\t" + dif + "\t" + getSymbol(ids[0]) + "\t" + getSymbol(ids[1]) + "\t" + p + "\n");
                } catch (IOException ex) {
                    Logger.getLogger(GenePairTest.class.getName()).log(Level.SEVERE, null, ex);
                }

            }
        }
        try {
            writer.close();
        } catch (IOException ex) {
            Logger.getLogger(GenePairTest.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    public void runRandomTest(int iterations, OntologyType ontologyType) {

        this.ontologyType = ontologyType;
        OboParser oboParser = new OboParser();
        Ontology ontology_mf = oboParser.getOntology(oboFile, ontologyType);
        GoaParser goaParser = new GoaParser(goaFile, ontology_mf);
        Map<String, AnnotatedGene> annotatedGenes = goaParser.getAnnotatedGenes();

        BufferedWriter writer = FileUtils.getBufferedWriter(randFile);
        MinetResultsParser minetResultsParser;
        difs = new double[iterations];
        try {
            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_AL = minetResultsParser.getResultsFromFile(MonkeyType.A, CellType.L);
            probeIds_AL = minetResultsParser.getProbeIds();

            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_AP = minetResultsParser.getResultsFromFile(MonkeyType.A, CellType.P);
            probeIds_AP = minetResultsParser.getProbeIds();

            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_ML = minetResultsParser.getResultsFromFile(MonkeyType.M, CellType.L);
            probeIds_ML = minetResultsParser.getProbeIds();

            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_MP = minetResultsParser.getResultsFromFile(MonkeyType.M, CellType.P);
            probeIds_MP = minetResultsParser.getProbeIds();
        } catch (IOException ex) {
            ex.printStackTrace();
        }

        nonredundantSwissIds = getNonRedundantSwissIds();
        swissToProbe = getSwissIdToProbeId();
        probeToSwiss = getProbeIdToSwiss();
        goTermMap = getGoTermMap(annotatedGenes);
        
        List<String> probeIds_intersect = new ArrayList<String>();
        probeIds_intersect.addAll(probeIds_AL);
        probeIds_intersect.retainAll(probeIds_AP);
        probeIds_intersect.retainAll(probeIds_ML);
        probeIds_intersect.retainAll(probeIds_MP);
        probeIds_intersect = filterList(probeIds_intersect);
        System.out.println(probeIds_intersect.size());

        MersenneTwisterRNG twister = new MersenneTwisterRNG();
        for (int z = 0; z < iterations; z++) {


            double[] mi_A = {minetResults_AL.get(getRandomKey(twister, probeIds_intersect)), minetResults_AP.get(getRandomKey(twister, probeIds_intersect))};
            double[] mi_M = {minetResults_ML.get(getRandomKey(twister, probeIds_intersect)), minetResults_MP.get(getRandomKey(twister, probeIds_intersect))};
            double mean_A = (mi_A[0] + mi_A[1]) / 2;
            double mean_M = (mi_M[0] + mi_M[1]) / 2;
            double dif = 0;
            if (mean_A > mean_M) {
                dif = mean_A - mean_M;
            } else {
                dif = mean_M - mean_A;
            }
            try {
                writer.append(mean_A + "\t" + mean_M + "\t" + dif + "\n");
            } catch (IOException ex) {
                Logger.getLogger(GenePairTest.class.getName()).log(Level.SEVERE, null, ex);
            }
            difs[z] = dif;

        }
        try {
            writer.close();
        } catch (IOException ex) {
            Logger.getLogger(GenePairTest.class.getName()).log(Level.SEVERE, null, ex);
        }

    }

    private String getRandomKey(MersenneTwisterRNG twister, List<String> probeIds_intersect) {
        int i = twister.nextInt(probeIds_intersect.size());
        int j = twister.nextInt(probeIds_intersect.size());
        String[] ids = {probeIds_intersect.get(i), probeIds_intersect.get(j)};
        Arrays.sort(ids);
        return ids[0] + ":" + ids[1];

    }

    public void runTest(OntologyType ontologyType, int iterations) {
        this.ontologyType = ontologyType;
        OboParser oboParser = new OboParser();
        Ontology ontology_mf = oboParser.getOntology(oboFile, ontologyType);
        GoaParser goaParser = new GoaParser(goaFile, ontology_mf);
        Map<String, AnnotatedGene> annotatedGenes = goaParser.getAnnotatedGenes();
        MinetResultsParser minetResultsParser;
        BufferedWriter writer_real = FileUtils.getBufferedWriter(goResultsFile);

        try {
            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_AL = minetResultsParser.getResultsFromFile(MonkeyType.A, CellType.L);
            probeIds_AL = minetResultsParser.getProbeIds();

            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_AP = minetResultsParser.getResultsFromFile(MonkeyType.A, CellType.P);
            probeIds_AP = minetResultsParser.getProbeIds();

            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_ML = minetResultsParser.getResultsFromFile(MonkeyType.M, CellType.L);
            probeIds_ML = minetResultsParser.getProbeIds();

            minetResultsParser = new MinetResultsParser();
            minetResultsParser.setMiThreshold(0);
            minetResults_MP = minetResultsParser.getResultsFromFile(MonkeyType.M, CellType.P);
            probeIds_MP = minetResultsParser.getProbeIds();
        } catch (IOException ex) {
            ex.printStackTrace();
        }

        nonredundantSwissIds = getNonRedundantSwissIds();
        swissToProbe = getSwissIdToProbeId();
        probeToSwiss = getProbeIdToSwiss();
        goTermMap = getGoTermMap(annotatedGenes);


        List<String> probeIds_intersect = new ArrayList<String>();
        probeIds_intersect.addAll(probeIds_AL);
        probeIds_intersect.retainAll(probeIds_AP);
        probeIds_intersect.retainAll(probeIds_ML);
        probeIds_intersect.retainAll(probeIds_MP);
        System.out.println(probeIds_intersect.size());

        TreeMap<GoTermPair, List<Double>> realMiDifValues = new TreeMap<GoTermPair, List<Double>>();
        TreeMap<GoTermPair, List<Double>> meanValues_A = new TreeMap<GoTermPair, List<Double>>();
        TreeMap<GoTermPair, List<Double>> meanValues_M = new TreeMap<GoTermPair, List<Double>>();

        for (int i = 0; i < probeIds_intersect.size(); i++) {
            for (int j = i + 1; j < probeIds_intersect.size(); j++) {
                String[] ids = {probeIds_intersect.get(i), probeIds_intersect.get(j)};
                Arrays.sort(ids);
                String key = ids[0] + ":" + ids[1];
                double[] mi_A = {minetResults_AL.get(key), minetResults_AP.get(key)};
                double[] mi_M = {minetResults_ML.get(key), minetResults_MP.get(key)};
                double mean_A = (mi_A[0] + mi_A[1]) / 2;
                double mean_M = (mi_M[0] + mi_M[1]) / 2;
                double dif = 0;
                if (mean_A > mean_M) {
                    dif = mean_A - mean_M;
                } else {
                    dif = mean_M - mean_A;
                }

                List<GoTerm> terms1 = getGoTerms(probeIds_intersect.get(i));
                List<GoTerm> terms2 = getGoTerms(probeIds_intersect.get(j));
                List<GoTermPair> termPairs = getTermPairs(terms1, terms2);
                for (GoTermPair gtp : termPairs) {
                    if (!realMiDifValues.containsKey(gtp)) {
                        realMiDifValues.put(gtp, new ArrayList<Double>());
                        meanValues_A.put(gtp, new ArrayList<Double>());
                        meanValues_M.put(gtp, new ArrayList<Double>());
                    }
                    realMiDifValues.get(gtp).add(dif);
                    meanValues_A.get(gtp).add(mean_A);
                    meanValues_M.get(gtp).add(mean_M);

                }

            }
        }
        List<Double> allMiDifValues = new ArrayList<Double>();
        List<GoTermPair> toRemove = new ArrayList<GoTermPair>();
        for (GoTermPair gtp : realMiDifValues.keySet()) {
            if (realMiDifValues.get(gtp).size() >= minNoMiValuesPerTermPair) {
                try {
                    writer_real.append(gtp.getUniqueId() + "\t" + gtp.getGoTerm1().getName() + "\t" + gtp.getGoTerm2().getName() + "\t" + realMiDifValues.get(gtp).size() + "\t" + getMean(realMiDifValues.get(gtp)) + "\t"+getMean(meanValues_A.get(gtp))+"\t"+getMean(meanValues_M.get(gtp))+"\n");
                } catch (IOException ex) {
                    Logger.getLogger(GenePairTest.class.getName()).log(Level.SEVERE, null, ex);
                }
                for (double mi : realMiDifValues.get(gtp)) {
                    allMiDifValues.add(mi);
                }
            } else {
                toRemove.add(gtp);
            }
        }
        try {
            writer_real.close();
        } catch (IOException ex) {
            Logger.getLogger(GenePairTest.class.getName()).log(Level.SEVERE, null, ex);
        }
        for (GoTermPair gtp : toRemove) {
            realMiDifValues.remove(gtp);
        }

        MersenneTwisterRNG twister = new MersenneTwisterRNG();
        System.out.println("finished outputting actual go term pair values");
        for (int i = 0; i < iterations; i++) {
            BufferedWriter writer_model = FileUtils.getBufferedWriter(new File(goResultsFile_model, "iteration_" + i + ".txt"));
            Collections.shuffle(allMiDifValues, twister);
            TreeMap<GoTermPair, List<Double>> modelMiDifValues = getModelDifValues(realMiDifValues, allMiDifValues);
            for (GoTermPair gtp : modelMiDifValues.keySet()) {
                try {
                    writer_model.append(gtp.getUniqueId() + "\t" + modelMiDifValues.get(gtp).size() + "\t" + getMean(modelMiDifValues.get(gtp)) + "\n");
                    writer_model.flush();
                } catch (IOException ex) {
                    Logger.getLogger(GenePairTest.class.getName()).log(Level.SEVERE, null, ex);
                }
            }
            System.out.println("finished iteration " + i);
            try {
                writer_model.close();
            } catch (IOException ex) {
                Logger.getLogger(GenePairTest.class.getName()).log(Level.SEVERE, null, ex);
            }
        }

    }

    public void outputGoTermPairPValues(int iterations) throws IOException {
        BufferedReader reader_real = FileUtils.getBufferedReader(goResultsFile);
        String line;
        HashMap<String, Double> difHash = new HashMap<String, Double>();

        while ((line = reader_real.readLine()) != null) {
            String[] split = line.split("\t");
            String key = split[0];
            double avDif = Double.parseDouble(split[4]);
            difHash.put(key, avDif);
        }
        reader_real.close();

        HashMap<String, List<Double>> modelDifHash = new HashMap<String, List<Double>>();
        for (int i = 0; i < iterations; i++) {
            File f = new File(goResultsFile_model, "iteration_" + i + ".txt");
            System.out.println("reading file: " + f.getName());
            BufferedReader reader = FileUtils.getBufferedReader(f);
            while ((line = reader.readLine()) != null) {
                String[] split = line.split("\t");
                String key = split[0];
                double avDif = Double.parseDouble(split[2]);
                if (i == 0) {
                    modelDifHash.put(key, new ArrayList<Double>());
                }
                modelDifHash.get(key).add(avDif);
            }
        }
        BufferedWriter writer = FileUtils.getBufferedWriter(goResultsFile_pvalues);
        reader_real = FileUtils.getBufferedReader(goResultsFile);
        while ((line = reader_real.readLine()) != null) {
            String[] split = line.split("\t");
            String key = split[0];
            double dif = difHash.get(key);
            List<Double> modelDifValues = modelDifHash.get(key);
            Collections.sort(modelDifValues);
            double p = getP(modelDifValues, dif);
            writer.append(key + "\t" + split[1] + "\t" + split[2] + "\t" + split[3] + "\t" + split[4] + "\t" +split[5]+"\t"+split[6]+"\t"+ p + "\n");
        }
        writer.close();
    }

    

    private TreeMap<String, List<GoTerm>> getGoTermMap(Map<String, AnnotatedGene> annotatedGenes) {
        TreeMap<String, List<GoTerm>> termMap = new TreeMap<String, List<GoTerm>>();


        for (String swissId : nonredundantSwissIds) {
            if (annotatedGenes.containsKey(swissId)) {
                AnnotatedGene ag = annotatedGenes.get(swissId);
                List<GoTerm> terms = ag.getAllTermsIncludingParents();
                for (GoTerm term : terms) {
                    if (!termMap.containsKey(swissId)) {
                        termMap.put(swissId, new ArrayList<GoTerm>());
                    }
                    termMap.get(swissId).add(term);
                }
            }
        }
        return termMap;
    }

    private TreeSet<String> getNonRedundantSwissIds() {
        TreeSet<String> swissIdsSet = new TreeSet<String>();
        List[] probeIdSets = {probeIds_AL, probeIds_AP, probeIds_ML, probeIds_MP};
        for (List probeIds : probeIdSets) {
            List<String> probeIdsParam = (List<String>) probeIds;
            for (String probeId : probeIdsParam) {
                List<String> swissIds = getSwissIds(probeId);
                for (String swissId : swissIds) {
                    if (!swissIdsSet.contains(swissId)) {
                        swissIdsSet.add(swissId);
                    }
                }
            }
        }
        return swissIdsSet;
    }

    private Map<String, List<String>> getSwissIdToProbeId() {
        HashMap<String, List<String>> lookup = new HashMap<String, List<String>>();
        for (String swissId : nonredundantSwissIds) {
            if (!lookup.containsKey(swissId)) {
                lookup.put(swissId, new ArrayList<String>());
            }
            lookup.get(swissId).addAll(getProbeIds(swissId));
        }
        return lookup;
    }

    private Map<String, List<String>> getProbeIdToSwiss() {
        String sql = "select distinct swissProtAccession, probeId from probeIdLookup t1, swissProtLookup t2 where t1.gId=t2.gId";
        Map<String, List<String>> ids = new HashMap<String, List<String>>();
        ResultSet rs = handle.runSQLquery(sql);
        try {
            while (rs.next()) {
                String probeId = rs.getString("probeId");
                String swissId = rs.getString("swissProtAccession");
                if (!ids.containsKey(probeId)) {
                    ids.put(probeId, new ArrayList<String>());
                }
                ids.get(probeId).add(swissId);
            }
            rs.close();
        } catch (SQLException ex) {
            ex.printStackTrace();
        }
        return ids;
    }

    private List<String> getSwissIds(String probeIdString) {
        String sql = "select distinct swissProtAccession from probeIdLookup t1, swissProtLookup t2 where t1.gId=t2.gId and t1.probeId=" + probeIdString;
        List<String> swissIds = new ArrayList<String>();
        ResultSet rs = handle.runSQLquery(sql);
        try {
            while (rs.next()) {
                String probeId = rs.getString("swissProtAccession");
                swissIds.add(probeId);
            }
            rs.close();
        } catch (SQLException ex) {
            ex.printStackTrace();
        }
        return swissIds;
    }

    private List<String> getProbeIds(String swissIdString) {
        String sql = "select distinct probeId from probeIdLookup t1, swissProtLookup t2 where t1.gId=t2.gId and t2.swissProtAccession='" + swissIdString + "'";
        List<String> probeIds = new ArrayList<String>();
        ResultSet rs = handle.runSQLquery(sql);
        try {
            while (rs.next()) {
                String probeId = rs.getString("probeId");
                probeIds.add(probeId);
            }
            rs.close();
        } catch (SQLException ex) {
            ex.printStackTrace();
        }
        return probeIds;
    }

    private String getGoTermsString(String probeId) {
        List<String> swissIds = getSwissIds(probeId);
        List<GoTerm> terms = new ArrayList<GoTerm>();
        for (String swissId : swissIds) {
            List<GoTerm> theseTerms = goTermMap.get(swissId);
            if (theseTerms != null) {
                for (GoTerm gt : theseTerms) {
                    if (!terms.contains(gt)) {
                        terms.add(gt);
                    }
                }
            }
        }
        StringBuffer sb = new StringBuffer();
        sb.append("");
        for (GoTerm gt : terms) {
            sb.append(gt.getId() + ":" + gt.getName() + ",");
        }

        return sb.toString();
    }

    private String getSymbol(String probeId) {
        String sql = "select symbol from ncbi.gene_info t1, probeIdLookup t2 where t1.gId=t2.gId and t2.probeId=" + probeId;
        ResultSet rs = handle.runSQLquery(sql);
        String symbol = probeId;
        try {
            if (rs.next()) {
                symbol = rs.getString("symbol");
            }
            rs.close();
        } catch (SQLException ex) {
            ex.printStackTrace();
        }
        return symbol;

    }

    private double getP(List<Double> difs, double dif) {
        for (int d = 0; d < difs.size(); d++) {

            if (dif <= difs.get(d)) {
//                System.out.println(dif+"\t"+difs.get(d)+"\t"+(difs.size() - d + 0.0) / difs.size());
                return (difs.size() - d + 0.0) / difs.size();
            }
        }
        return 1.0 / difs.size();
    }

    private double getP(double[] difs, double dif) {
        for (int d = 0; d < difs.length; d++) {
            if (dif <= difs[d]) {
                return (difs.length - d + 0.0) / difs.length;
            }
        }
        return 1.0 / difs.length;
    }

    private List<GoTerm> getGoTerms(String probeId) {
        List<GoTerm> terms = new ArrayList<GoTerm>();
        List<String> swissIds = probeToSwiss.get(probeId);
        if (swissIds != null) {
            for (String swissId : swissIds) {
                List<GoTerm> theseTerms = goTermMap.get(swissId);
                if (theseTerms != null) {
                    for (GoTerm thisTerm : theseTerms) {
                        if (!terms.contains(thisTerm) && include(thisTerm)) {
//                            System.out.println(thisTerm.getName());
                            terms.add(thisTerm);
                        }
                    }
                }
            }
        }
        return terms;


    }

    private List<GoTermPair> getTermPairs(List<GoTerm> terms1, List<GoTerm> terms2) {
        List<GoTermPair> termPairs = new ArrayList<GoTermPair>();
        for (GoTerm term1 : terms1) {
            for (GoTerm term2 : terms2) {
                termPairs.add(new GoTermPair(term1, term2));
            }
        }
        return termPairs;
    }

    private double getMean(List<Double> doubles) {
        double sum = 0;
        for (double d : doubles) {
            sum += d;
        }
        return sum / doubles.size();
    }

    private TreeMap<GoTermPair, List<Double>> getModelDifValues(TreeMap<GoTermPair, List<Double>> realMiDifValues, List<Double> shuffledMiDifValues) {
        TreeMap<GoTermPair, List<Double>> modelValues = new TreeMap<GoTermPair, List<Double>>();
        int index = 0;
        for (GoTermPair gtp : realMiDifValues.keySet()) {
            modelValues.put(gtp, new ArrayList<Double>());
            for (int i = 0; i < realMiDifValues.get(gtp).size(); i++) {
                modelValues.get(gtp).add(shuffledMiDifValues.get(index + i));
            }
            index = index + realMiDifValues.get(gtp).size();
        }
        return modelValues;
    }

    private boolean include(GoTerm thisTerm) {
        if (!limitToImmuneAndCellCycle) {
            return true;
        } else {

            if (thisTerm.isDescendedFrom(new GoTerm(7049))) {

                return true;
            }
            if (thisTerm.isDescendedFrom(new GoTerm(6955))) {

                return true;
            }

//            if (thisTerm.isDescendedFrom(new GoTerm(34340))) {
//
//                return true;
//            }


            return false;
        }
    }

    private List<String> filterList(List<String> probeIds) {
        if(limitToImmuneAndCellCycle){
            List<String> newList = new ArrayList<String>();
            for(String probeId : probeIds){
                boolean include = false;
                List<String> swissIds = probeToSwiss.get(probeId);
                if(swissIds!=null){
                    for(String swissId : swissIds){
                        List<GoTerm> terms = goTermMap.get(swissId);
                        if(terms!=null){
                            for(GoTerm gt : terms){
                                if(include(gt)){
                                    include = true;
                                }
                            }
                        }
                    }
                }
                if(include){
                    newList.add(probeId);
                }
            }
            return newList;
        } else {
            return probeIds;
        }

    }
}
